/*
 * EvolutionProcess.java
 *
 * Created on den 23 oktober 2007, 11:22
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */
package grex;

import grex.Data.ArffTableModel;
import WekaModels.*;
import grex.DT.tree.DecisionTreeBuilder;
import grex.Data.BootStrapArffTableModel;
import grex.Data.ExtractionArffTableModel;

import grex.Graphics.PerformancePanel;

import grex.fitnessfunctions.Classification.MultipleSolutionsFitness;
import grex.fitnessfunctions.ErrorFunctions.ErrorCalculator;
import grex.fitnessfunctions.ErrorFunctions.IErrorFunction;
import grex.fitnessfunctions.Classification.ACCFitness;
import grex.fitnessfunctions.Classification.AUCFitness;
import grex.fitnessfunctions.FitnessFunction;
import grex.genes.GeneException;

import java.io.IOException;
import java.io.Serializable;
import java.text.NumberFormat;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import wekaGrexBridge.J48Injection;

import wekaGrexBridge.WekaArffTableModel;

/**
 *
 * @author RIK
 */
public class EvolutionProcess implements Runnable, Serializable {

    ArrayList<EvolutionListener> evolutionListeners = new ArrayList<EvolutionListener>();
    CopyOnWriteArrayList<SimplificationListener> simplificationListeners = new CopyOnWriteArrayList<SimplificationListener>();
    ArrayList<GP> batchGPs, selectedGPs;
    HashMap<String, ArrayList<IPredictiveModel>> predictiveModels = new HashMap<String, ArrayList<IPredictiveModel>>();
    private IPredictiveModel simplificationModel;
    
    public void setSimplificationModel(IPredictiveModel model){
        simplificationModel = model;
    }
    
    public ArrayList<GP> getBatchGPs() {
        return batchGPs;
    }

    public void setBatchGPs(ArrayList<GP> batchGPs) {
        this.batchGPs = batchGPs;
    }
    private Options options;
    Simplyfier simplyfier = null;
    GP bestGP, bestSimplyfiedGP, publicGP, bestSelectedGP;
    private int currentGeneration;
    int currentBatch, persistence = 0;
    Hashtable<String, EvolutionResult> foldResults, selectedFoldResults;
    World world;
    Thread t;
    private ArffTableModel data;
    private String summary = "", selectedSummary = "";
    NumberFormat nf = NumberFormat.getInstance(Locale.ENGLISH);
    GP batchWinner;
    ErrorCalculator ec;
    private Environment environment;
    private final int TREE_INJECTION_INTERVAL = 5,
            NR_OF_INJECTED_MUTADED_TREES = 5;

    /**
     * Creates a new instance of EvolutionProcess
     */
    public EvolutionProcess(ArffTableModel data, Options options) {
        try {
            options.setNR_OF_INPUT_COLUMNS(data.getNrOfCategorialColumns() + data.getNrOfContinousColumns());
            this.data = data;
            world = new World();
            this.options = options;
            nf.setMinimumFractionDigits(4);
            //   this.addEvolutionListener(new ConsoleOutputHandler());
        } catch (BNFException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (IOException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
    }

    public Thread getThread() {
        return t;
    }

    public synchronized void addEvolutionListener(EvolutionListener l) {
        evolutionListeners.add(l);
    }

    public synchronized void removeEvolutionLister(EvolutionListener l) {
        evolutionListeners.remove(l);
    }

    public synchronized void addSimplificationListener(SimplificationListener l) {
        simplificationListeners.add(l);
    }

    public synchronized void removeallEvoltuionListeners() {
        evolutionListeners.clear();
    }

    public synchronized void removeSimplificationListener(SimplificationListener l) {
        simplificationListeners.remove(l);
    }

    public void startEvolution(Options options) {
        this.options = options;
        t = new Thread(this);
        t.start();
    }

    public void startEvolution() {
        t = new Thread(this);
        t.start();
    }

    public void startEvolution(String dataFile, String BNFfile) {
        try {
            t = new Thread(this);

            environment = new Environment(dataFile, BNFfile);
            this.options = environment.getOptions();
            this.data = environment.getData();
            getOptions().setNR_OF_INPUT_COLUMNS(getData().getNrOfCategorialColumns() + getData().getNrOfContinousColumns());
            world.createNewPopulation(new Environment(getOptions()));
            t.start();
        } catch (BNFException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (IOException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
    }

    public boolean isAlive() {
        return t.isInterrupted();
    }

    private void addPredicitiveModel(IPredictiveModel model) {
        ArrayList<IPredictiveModel> models = predictiveModels.get(model.getName());
        if (models == null) {
            models = new ArrayList<IPredictiveModel>();
            predictiveModels.put(model.getName(), models);
        }
        models.add(model);
        if (!modelNames.contains(model.getName())) {
            modelNames.add(model.getName());
        }
    }
    String lastTestFold = "";
    ResultSaver resultSaver;
    ResultSaverTest resultSaverTest;
    ArrayList<String> modelNames = new ArrayList<String>();
    private ArffTableModel orgData;

    private void initBenchmarkTechniques(String fold) {
        if (options.getPROBLEM_TYPE() == Options.REGRESSION) {
            if (options.isZeroRReg()) {
                GrexZeroR zero = new GrexZeroR(orgData);
                zero.initFold(fold);
                zero.train();
                addPredicitiveModel(zero);
            }
            //  */
            if (options.isSVMReg()) {
                GrexSVMReg svmReg = new GrexSVMReg(orgData);
                svmReg.initFold(fold);
                svmReg.train();
                addPredicitiveModel(svmReg);//*/
            }

            if (options.isRBF()) {
                GrexRBF rbf = new GrexRBF(orgData);
                rbf.initFold(fold);
                rbf.train();
                addPredicitiveModel(rbf);//*/
            }

            if (options.isM5PModelTree()) {
                GrexM5P m5P = new GrexM5P(orgData);
                m5P.initFold(fold);
                m5P.train();
                addPredicitiveModel(m5P);//*/
            }

            if (options.isM5PRegTree()) {
                GrexM5PRegTree m5P = new GrexM5PRegTree(orgData);
                m5P.initFold(fold);
                m5P.train();
                addPredicitiveModel(m5P);//*/
            }

            if (options.isMLPReg()) {
                GrexMLP mlp = new GrexMLP(orgData);
                mlp.initFold(fold);
                mlp.train();
                addPredicitiveModel(mlp);//*/
            }
            if (options.isReptreeREG()) {
                GrexREPTree tree = new GrexREPTree(orgData);
                tree.initFold(fold);
                tree.train();
                addPredicitiveModel(tree);//*/
            }

            if (options.isLinearRegression()) {
                GrexLinearRegression reg = new GrexLinearRegression(orgData);
                reg.initFold(fold);
                reg.train();
                addPredicitiveModel(reg);
            }

            if (options.isPaceRegression()) {
                GrexPaceRegression pace = new GrexPaceRegression(orgData);

                pace.initFold(fold);
                pace.train();
                addPredicitiveModel(pace);
            }

            if (options.isBaggedM5P()) {
                GrexM5PBag bag = new GrexM5PBag(orgData);
                bag.initFold(fold);
                bag.train();
                addPredicitiveModel(bag);
            }
            /*
             * GrexWekaEnsemble ens = new GrexWekaEnsemble(orgData);
             * ens.initFold(fold); // (()) reg.train(); ens.addMember(m5P);
             * ens.addMember(tree); ens.addMember(reg); // ens.addMember(mlp);
             * addPredicitiveModel(ens); //
             */


        } else {//Classification           

            System.out.println("\n--------------------------------");
            if (options.isZeroR()) {
                GrexZeroR zero = new GrexZeroR(orgData);
                zero.initFold(fold);
                zero.train();
                addPredicitiveModel(zero);
            }

            if (options.isJ48()) {
                GrexJ48 j48 = new GrexJ48(orgData);
                j48.initFold(fold);
                j48.train();
                addPredicitiveModel(j48);
            }

            if (options.isSimpleCart()) {
                GrexSimpleCart scart = new GrexSimpleCart(orgData);
                scart.initFold(fold);
                scart.train();
                addPredicitiveModel(scart);
            }

            if (options.isRandomForest()) {
                GrexRandomForest rf = new GrexRandomForest(orgData);
                rf.initFold(fold);
                rf.train();
                addPredicitiveModel(rf);
            }

            if (options.isMLP()) {
                GrexMLP mlp = new GrexMLP(orgData);
                mlp.initFold(fold);
                mlp.train();
                addPredicitiveModel(mlp);
            }
            if (options.isSVM()) {
                GrexSVMClass svm = new GrexSVMClass(orgData);
                svm.initFold(fold);
                svm.train();
                addPredicitiveModel(svm);
            }
        }
    }
    
    

    public void run() {
        try {
            orgData = data;

            // Rule Extraction
            
            if (options.isRULE_EXTRACT_FROM_RF() && options.getPROBLEM_TYPE() == Options.CLASSIFICATION) {
                WekaPredictiveModel model = new GrexRandomForest(data);
                ExtractionArffTableModel extractionData = new ExtractionArffTableModel(data, model);
                extractionData.setUseOracle(options.isORACLE_COACHING_TEST());
                data = extractionData;
            }

            if (options.isRULE_EXTRACTION_FROM_M5PBag() && options.getPROBLEM_TYPE() == Options.REGRESSION) {
                WekaPredictiveModel model = new GrexM5PBag(data);
                ExtractionArffTableModel extractionData = new ExtractionArffTableModel(data, model);
                extractionData.setUseOracle(options.isORACLE_COACHING_TEST());
                data = extractionData;
            }


            //Bootstraping
          /*
             * orgData = data; BootStrapArffTableModel bagData = new
             * BootStrapArffTableModel(data); data = bagData;//
             */




            String startFold = getOptions().getTestFOLD();
            summary = "";
            foldResults = new Hashtable<String, EvolutionResult>();
            selectedFoldResults = new Hashtable<String, EvolutionResult>();
            environment = new Environment(getData(), getOptions());
//            environment.getFitnessFunctionFactory().clearFitnessFunction();
//            FitnessFunction fitness = environment.getFitnessFunctionFactory().getFitnessFunction();
            ec = new ErrorCalculator(environment);
          
            // fitness.setAlternatePressure(true);
 //           addEvolutionListener(fitness);
            addEvolutionListener(resultSaver);


/*
            if (options.getFITNESS_FUNCTION().equalsIgnoreCase("MultipleSolutionsFitness")) {
                resultSaver = new ResultSaver();
                //  resultSaverTest = new ResultSaverTest();
                addEvolutionListener(resultSaver);
                addEvolutionListener(resultSaverTest);
                this.addEvolutionListener((EvolutionListener) environment.getFitnessFunctionFactory().getFitnessFunction());
            }
            if (options.getFITNESS_FUNCTION().equalsIgnoreCase("TombolaFitness")) {
                this.addEvolutionListener((EvolutionListener) environment.getFitnessFunctionFactory().getFitnessFunction());
            }*/

            ArrayList<String> arlFolds;
            if (getOptions().getTestFOLD().equalsIgnoreCase("ALL")) {
                arlFolds = environment.getFolds();
            } else {
                arlFolds = new ArrayList<String>();
                arlFolds.add(getOptions().getTestFOLD());
            }

            long startTime = System.currentTimeMillis();
            double orgLengthPunishment = options.getPUNISHMENT_FOR_LENGTH();
            double originalTombolaPercent = options.getTOMBOLA_PERCENT();
            for (int f = 0; f < arlFolds.size(); f++) {

                if (t.isInterrupted()) {
                    break;
                }
                String fold = arlFolds.get(f);
                batchGPs = new ArrayList<GP>();
                selectedGPs = new ArrayList<GP>();
                getOptions().setTestFOLD(fold);


                getOptions().setWinners(null);
                environment.initFold(fold);

                if (options.isEXP_VAL_FOLDS()) {
                    int valFoldIndex = f + 1;
                    if (valFoldIndex == arlFolds.size()) {
                        valFoldIndex = 0;
                    }
                    String valFold = arlFolds.get(valFoldIndex);
                    options.setVAL_FOLD(valFold);
                    orgData.setValFold(valFold);
                    data.setValFold(valFold);
                }

                //PredictiveEnsemble pEnsemble = new PredictiveEnsemble(environment);
               // addPredicitiveModel(pEnsemble);


                initBenchmarkTechniques(fold);





                if (options.isRUN_GREX()) {

                    int batchSize = getOptions().getBATCH_SIZE();
                    int nrOfgenerations = getOptions().getGENERATIONS();
                    ArrayList<GP> injectionGPs = null;
                    double injectionGPAvgOutOfBagAcc = 0, injectionGPBootStrapAcc = 0;
                    double injectionGPReasonableACC = 0;
                    int maxTreeSize = options.getMAX_TREE_SIZE();
                    for (currentBatch = 0; currentBatch < batchSize && !t.isInterrupted(); currentBatch++) {
                        if (t.isInterrupted()) {
                            break;
                        }
                        options.setPUNISHMENT_FOR_LENGTH(orgLengthPunishment);
                        if (getOptions().isEVOLVE_RANDOMFOREST()) {
                            GP[] population = new GP[getOptions().getPOPULATION_SIZE()];
                            for (int p = 0; p < population.length; p++) {
                                population[p] = new GPEnsemble(environment, getOptions().getRANDOM_FOREST_SIZE());
                            }
                            world.createNewPopulation(population, environment);
                        } else if (getOptions().isUSE_GP_INJECTION()) {
                           //if (getData() instanceof WekaArffTableModel) {
                           
                            
                                WekaArffTableModel m = new WekaArffTableModel(new J48Injection(),orgData);// (WekaArffTableModel) getData();                               
                                injectionGPReasonableACC = m.calcReasonableAcc(10, 0.1f,fold);
                                injectionGPs = new ArrayList<GP>();
                                for (int i = 1; i < options.getGENERATIONS() / TREE_INJECTION_INTERVAL + 3; i++) {
                                    GP j48 = m.createJ48GP(0.05f, environment,fold);
                                    //         System.out.println("1Length: " + j48.getLength() + "Acc" + j48.getTrainAcc());
                                    j48.cutTree(i);
                                    j48.train();
                                   // j48.calcNrOfNodes();
                                    System.out.println("Length: " + j48.getNrOfNodes() + "Acc" + j48.calcTrainError("ACC"));// errorCalculator.getACC().calcError(j48, j48.pcTrain));// j48.getTrainAcc());
                                    //  j48.train();
                                    injectionGPs.add(0, j48); // to be compatible with jaTI that sorts the tre in descending order.
                                    //  j48.train();
                                    //  j48.optimize();
                                    j48.getHead().notifyChildrenOfModification();                                    
                                }
                         /*   } else {
                                DecisionTreeBuilder injectionGPbuilder = new DecisionTreeBuilder(environment, false);
                                injectionGPbuilder.buildBootStrapTrees(getOptions().getGENERATIONS() / 10);
                                injectionGPBootStrapAcc = injectionGPbuilder.getBootstrapACC();
                                injectionGPAvgOutOfBagAcc = injectionGPbuilder.getOutofBagAvgACC();
                                injectionGPReasonableACC = (injectionGPBootStrapAcc + injectionGPAvgOutOfBagAcc) / 2;
                                injectionGPbuilder.buildAscendingBootStrapTrees(options.getGENERATIONS() / TREE_INJECTION_INTERVAL, 0.05, 0.001);
                                injectionGPs = injectionGPbuilder.getBootStrapTreesSsArrayList();
                            }//*/
                            injectionGPReasonableACC = (1-injectionGPReasonableACC) * 100;
                            getOptions().setTARGET_ACCURACY(injectionGPReasonableACC);
                            world.createNewPopulation(environment);
                        } else if (getOptions().isSTART_FROM_RANDOMFOREST()) {
                            DecisionTreeBuilder builder = new DecisionTreeBuilder(environment, false, 0.00001, 0.00001, 1);
                            builder.buildRandomForrest(getOptions().getRANDOM_FOREST_SIZE());
                            world.createNewPopulation(builder.getForrestAsGPs(), environment);//*/
                        } else if (getOptions().isBUILD_DECISIONTREE()) {
                            DecisionTreeBuilder builder = new DecisionTreeBuilder(environment, false, getOptions().getDT_SCORE_THRESHOLD(), getOptions().getDT_ENTROPY_THRESHOLD(), 1);
                            builder.buildDecisionTree();
                            world.createNewPopulation(getOptions().getPOPULATION_SIZE(), builder.getTreeAsGP(), environment);

                        } else {
                            world.createNewPopulation(environment);
                        }

                        persistence = options.getSTOP_WITHOUT_IMPROVMENT();
                        bestGP = world.getBestEverGP();
                        double bestFitness = bestGP.getFitness();
                        options.setTOMBOLA_PERCENT(originalTombolaPercent);

                        this.fireEvolutionStarted();

                        for (currentGeneration = 0; getCurrentGeneration() <= nrOfgenerations && !t.isInterrupted() && persistence != -1; currentGeneration++) {
                            if (t.isInterrupted()) {
                                break;
                            }
                            if (getCurrentGeneration() == nrOfgenerations) {
                                options.setTOMBOLA_PERCENT(1);
                            }
                            if (4 * getCurrentGeneration() < options.getGENERATIONS()) {
                                options.setMAX_TREE_SIZE(1000);
                            } else {
                                options.setMAX_TREE_SIZE(maxTreeSize);
                            }
                            bestGP = (GP) world.getPopulation().getBestGP().clone();
                            bestGP.calcAdjustedFitness();
                            //bestGP.optimize();
                            bestGP.clean();
                            world.evolve(1);
                            if (bestGP.getFitness() < bestFitness) {
                                persistence = options.getSTOP_WITHOUT_IMPROVMENT();
                                bestFitness = bestGP.getFitness();
                            }
                            this.fireGenerationTerminated();
                            if (options.getSTOP_WITHOUT_IMPROVMENT() != 0) {
                                persistence--;
                            }
                            //  bestGP.getFitnessFunction().


                            if (getOptions().isUSE_GP_INJECTION() && injectionGPs != null && (getCurrentGeneration()) % (TREE_INJECTION_INTERVAL * 2) == 0) {

                                if (bestGP.getTrainAccError() > injectionGPReasonableACC) {
                                    getOptions().setPUNISHMENT_FOR_LENGTH(getOptions().getPUNISHMENT_FOR_LENGTH() * 0.9);
                                    System.out.println("Decreased Length Punishmet:" + getOptions().getPUNISHMENT_FOR_LENGTH());
                                }
                            }

                            if (getOptions().isUSE_GP_INJECTION() && injectionGPs != null && getCurrentGeneration() % TREE_INJECTION_INTERVAL == 0) {
                                System.out.println("BestAcc: " + bestGP.getTrainAccError() + " ReasonableACC: " + injectionGPReasonableACC);
                                if (bestGP.getTrainAccError() > injectionGPReasonableACC) {
                                    if (injectionGPs.size() > 0) {
                                        GP gp = injectionGPs.get(injectionGPs.size() - 1);
                                        System.out.println("Injection!!!!! GP Length: " + gp.getNrOfNodes());
                                        world.getPopulation().injectGP(injectionGPs.get(injectionGPs.size() - 1), NR_OF_INJECTED_MUTADED_TREES);
                                        injectionGPs.remove(injectionGPs.size() - 1);
                                    }
 
                                }
                            }
                        }
                        batchWinner = (GP) world.getBestEverGP().clone();
                        batchWinner.clean();
                        batchGPs.add(batchWinner);


//                        pEnsemble.addPredictiveBaseModel(batchWinner);



                        //  selectedGPs.add(world.selectBestGP());
                        fireBatchTerminated();
                        //  ACCFitness.memmory = new Hashtable<String, Double>(50000);
                        //  ACCFitness.hashSaves=0;
                    }
                    //-----------------------------------
                    bestGP = getBestGpFromBatch(batchGPs);
                    addPredicitiveModel(bestGP);



                    //bestSelectedGP = getBestGpFromBatch(selectedGPs);
                    //  PersistenceGP.getInstance().saveGP(bestGP,"gp.grexGP");
                    if (getOptions().isSIMPLIFICATION()) {
                        startSimplification();
                    }
                    foldGPs.put(fold, batchGPs);
//                foldResults.put(fold, new EvolutionResult(batchGPs, getOptions().clone()));
                    //  selectedFoldResults.put(fold, new EvolutionResult(selectedGPs, getOptions().clone()));
                    lastTestFold = fold;
                    this.fireEvolutionTerminated();

                }//End RUN G-REX

                summary = getMeanResults() + getResults();//getStatSummary(arlFolds);
                getOptions().setTestFOLD(startFold);

            }
            

            fireEvolutionProcessTerminated();
            evolutionListeners.remove(resultSaver);

            System.out.println("Time: " + (System.currentTimeMillis() - startTime));
        } catch (Exception e) {
            e.printStackTrace();
            ErrorManager.getInstance().reportError(e);
            this.fireEvolutionTerminated();
        }

    }
    HashMap<String, ArrayList<GP>> foldGPs = new HashMap<String, ArrayList<GP>>();


    public GP getBatchWinner() {
        return batchWinner;
    }

    public HashMap<String, ArrayList<IPredictiveModel>> getPredicitveModels() {
        return predictiveModels;
    }

    public String getResults() {
        String results = "\n\n<------------ All results -------------->\n";
        ArrayList<String> keys;


        for (String key : modelNames) {
            try {
                ArrayList<IPredictiveModel> models = predictiveModels.get(key);
                results += key + "\n" + ec.getStatistics(models) + "\n\n";
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        return results;
    }

    public String getMeanResults() {
        String meanResults = options.getDATA_FILE() + " " + environment.getFolds().size() + "folds:";
        meanResults += "\t" + ec.getHeaders();
        for (String key : modelNames) {
            try {
                ArrayList<IPredictiveModel> models = predictiveModels.get(key);
                meanResults += key + "\t " + ec.getMeanStatistics(models) + "\n";
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        return meanResults + "\n";
    }
    
        public String getMeanResultsForFile() {
        String meanResults =""; 
       // meanResults += "\t" + ec.getHeaders();
        for (String key : modelNames) {
            try {
                ArrayList<IPredictiveModel> models = predictiveModels.get(key);
                meanResults +=options.getDATA_FILE() +"\t"+ key + "\t " + ec.getMeanStatistics(models) + "\n";
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        return meanResults;
    }

    public String getBatchResult() throws GeneException {

        String s = "";
        ArrayList<IPredictiveModel> models;
        ArrayList<IPredictiveModel> ipm = new ArrayList<IPredictiveModel>();
        for (String key : modelNames) {
            try {
                models = predictiveModels.get(key);
                s += ec.getStatisitcsAsRow(models);
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        StringTokenizer st = new StringTokenizer(options.getDATA_FILE(), "\\");
        String name = "";
        while (st.hasMoreTokens()) {
            name = st.nextToken();
        }
        return name + "\t" + s + "\n";
    }

    public String getBatchSensitivity() throws GeneException {

        String s = "";
        ArrayList<IPredictiveModel> models;
        ArrayList<IPredictiveModel> ipm = new ArrayList<IPredictiveModel>();
        for (String key : modelNames) {
            models = predictiveModels.get(key);
            double[] sensitivity = models.get(0).calcSensitivity();
            for (int i = 0; i < sensitivity.length; i++) {
                s += sensitivity[i] + " \t";
            }
            s += "\t";
        }
        StringTokenizer st = new StringTokenizer(options.getDATA_FILE(), "\\");
        String name = "";
        while (st.hasMoreTokens()) {
            name = st.nextToken();
        }
        return name + "\t" + s + "\n";
    }

    public String getBatchSensitivityHeaders() throws GeneException {
        String s = "";
        ArrayList<IPredictiveModel> models;
        for (String key : modelNames) {
            models = predictiveModels.get(key);
            String[] cNames = data.getColumnNames();
            for (int i = 0; i < cNames.length - 1; i++) {
                s += key + "-" + cNames[i] + " \t";
            }
            s += "\t";
        }
        return "Dataset" + "\t" + s + "\n";
    }

    public String getBatchResultPredictions() throws GeneException {


        StringTokenizer st = new StringTokenizer(options.getDATA_FILE(), "\\");
        String name = "";
        while (st.hasMoreTokens()) {
            name = st.nextToken();
        }

        String s = "Prediction for: " + name + "\n";
        ArrayList<IPredictiveModel> models;
        for (String key : modelNames) {
            models = predictiveModels.get(key);
            s += key + "\n";
            s += models.get(0).getOrderedTestPredictions();
        }

        return s + "\n";
    }

    public String getBatchHeaders() {
        return "\t" + ec.getStatisitcsAsRowHeaders();
    }

    public String getHeaders() {
        return "\t" + ec.getHeaders();
    }

    public String getStatSummary(ArrayList<String> folds) {
        String s = "";
        String treeResult = "";
        String bestInBatchResult = "Best in batch result:\n\t\t" + ec.getHeaders();
        String avgFoldResult = "Fold average result:\n\t\t" + ec.getHeaders();
        GP bestGP;
        double n = folds.size();
        try {
            ArrayList<IPredictiveModel> allGPs = new ArrayList<IPredictiveModel>();
            ArrayList<IPredictiveModel> bestGPs = new ArrayList<IPredictiveModel>();
            for (String fold : folds) {

                ArrayList<GP> gps = foldGPs.get(fold);
                allGPs.addAll(gps);
                avgFoldResult += fold + ":\t\t" + ec.getMeanStatistics(gps) + "\n";

                bestGP = getBestGpFromBatch(gps);
                bestGPs.add(bestGP);
                bestInBatchResult += fold + ":\t\t" + ec.getStatistics(bestGP) + "\n";

                treeResult += fold + " - Best GP in batch:" + "\n";
                treeResult += bestGP.getStatisticsAsString();
                treeResult += bestGP.getFormatedTree();
                treeResult += "\n\n";
                treeResult += "<-----------Test predictions----------->\n";
                treeResult += bestGP.getGPTestPrediction();
                treeResult += "<-----------Train predictions----------->\n";
                treeResult += bestGP.getGPTrainPrediction();
            }
            avgFoldResult += "AVG:\t" + ec.getMeanStatistics(allGPs);
            bestInBatchResult += "AVG:\t" + ec.getMeanStatistics(bestGPs);
            s += bestInBatchResult + "\n\n" + avgFoldResult + "\n\n" + treeResult + "\n\n" + options;


        } catch (Exception e) {
            ErrorManager.getInstance().reportError(e);
        }
        return s;
    }

    public String getStatSummary22(ArrayList<String> folds) {
        String avgs = "", stds = "", avgsSum = "", stdsSum = "", foldSumms = "", best = "", ens = "";
        double avgTrainACC = 0, avgTestACC = 0, avgTrainAUC = 0, avgTestAUC = 0, avgTrainBRE = 0, avgTestBRE = 0, avgLength = 0;
        double avgBestTrainACC = 0, avgBestTestACC = 0, avgBestTrainAUC = 0, avgBestTestAUC = 0, avgBestTrainBRI = 0, avgBestTestBRI = 0, avgBestLength = 0;
        double avgEnsTrainACC = 0, avgEnsTestACC = 0, avgEnsTrainAUC = 0, avgEnsTestAUC = 0, avgEnsTrainBRI = 0, avgEnsTestBRI = 0, avgEnsLength = 0;
        double stdBestTrainACC = 0, stdBestTestACC = 0, stdBestTrainAUC = 0, stdBestTestAUC = 0, stdBestTrainBRE = 0, stdBestTestBRE = 0, stdBestLength = 0;
        double stdTrainACC = 0, stdTestACC = 0, stdTrainAUC = 0, stdTestAUC = 0, stdTrainBRI = 0, stdTestBRI = 0, stdLength = 0;
        double[][] confusionMatrix = new double[getData().getNrOfTargetCategories()][getData().getNrOfTargetCategories()];
        NumberFormat nf = NumberFormat.getInstance();
        nf.setMaximumFractionDigits(4);
        GP gp;

        avgs += "Batch average result\n";
        avgs += "Fold\tAvgTrainACC\tAvgTestAcc\tAvgTrainAUC\tAvgTestAUC\tAvgTrainBRE\tAvgTestBRE\tAvgLength\n";
        stds += "Batch std\n";
        stds += "Fold\tStdTrainACC\tStdTestACC\tStdTrainAUC\tStdTestAUC\tStdTrainBRE\tStdTestBRE\tStdLength\n";
        best += "Best in batch result\n";
        best += "Fold\tTrainACC\tTestACC\tTrainAUC\tTestAUC\tTrainBRE\tTestBRE\tLength\n";
        ens += "Fold Ensemble Result\n";
        ens += "Fold\tTrainACC\tTestACC\tTrainAUC\tTestAUC\tTrainBRE\tTestBRE\tLength\n";
        try {
            double n = folds.size();
            for (String fold : folds) {

                EvolutionResult er = foldResults.get(fold);
                gp = er.getBestGpFromBatch();
                avgs += fold + "\t" + nf.format(er.getAvgTrainACC()) + " \t" + nf.format(er.getAvgTestACC()) + "\t" + nf.format(er.getAvgTrainAUC()) + "\t" + nf.format(er.getAvgTestAUC()) + "\t" + nf.format(er.getAvgTrainBrier()) + "\t" + nf.format(er.getAvgTestBrier()) + "\t" + nf.format(er.getAvgLength()) + "\n";

                stds += fold + "\t" + nf.format(er.getStdTrainACC()) + "\t" + nf.format(er.getStdTestACC()) + "\t" + nf.format(er.getStdTrainAUC()) + "\t" + nf.format(er.getStdTestAUC()) + "\t" + nf.format(er.getStdTrainBrier()) + "\t" + nf.format(er.getStdTestBrier()) + "\t" + nf.format(er.getStdLength()) + "\n";
                best += fold + "\t" + nf.format(gp.getTrainAccError()) + " \t" + nf.format(gp.getTestAccError()) + "\t" + nf.format(gp.getTrainAUCError()) + "\t" + nf.format(gp.getTestAUCError()) + "\t" + nf.format(gp.getTrainBRIERError()) + "\t" + nf.format(gp.getTestBRIERError()) + "\t" + nf.format(gp.getNrOfNodes()) + "\n";
                ens += fold + "\t" + nf.format(er.getEnsTrainACC()) + " \t" + nf.format(er.getEnsTestACC()) + "\t" + nf.format(er.getEnsTrainAUC()) + "\t" + nf.format(er.getEnsTestAUC()) + "\t" + nf.format(er.getEnsTrainBrier()) + "\t" + nf.format(er.getEnsTestBrier()) + "\n";
                avgTestACC += er.getAvgTestACC() / n;
                avgTestAUC += er.getAvgTestAUC() / n;
                avgTestBRE += er.getAvgTestBrier() / n;
                avgTrainACC += er.getAvgTrainACC() / n;
                avgTrainAUC += er.getAvgTrainAUC() / n;
                avgTrainBRE += er.getAvgTrainBrier() / n;
                avgLength += (1.0 / n) * er.getAvgLength();

                avgBestTestACC += gp.getTestAccError() / n;
                avgBestTestAUC += gp.getTestAUCError() / n;
                avgBestTestBRI += gp.getTestBRIERError() / n;
                avgBestTrainACC += gp.getTrainAccError() / n;
                avgBestTrainAUC += gp.getTrainAUCError() / n;
                avgBestTrainBRI += gp.getTrainBRIERError() / n;
                avgBestLength += (1.0 / n) * gp.getNrOfNodes();

                avgEnsTestACC += er.getEnsTestACC() / n;
                avgEnsTestAUC += er.getEnsTestAUC() / n;
                avgEnsTestBRI += er.getEnsTestBrier() / n;
                avgEnsTrainACC += er.getEnsTrainACC() / n;
                avgEnsTrainAUC += er.getEnsAVGTrainAUC() / n;
                avgEnsTrainBRI += er.getEnsTrainBrier() / n;

                foldSumms += "\n\n<----------------" + fold + "------------------>\n" + er.toString();


                //Calc confusionMatrix
                if (options.getPROBLEM_TYPE() == Options.CLASSIFICATION) {
                    for (Prediction p : gp.getPcTest().values()) {
                        confusionMatrix[(int) p.getTargetValue()][(int) p.getPrediction()]++;
                    }
                }
            }
            for (String fold : folds) {

                EvolutionResult er = foldResults.get(fold);
                gp = er.getBestGpFromBatch();
                stdTestACC += Math.pow(er.getAvgTestACC() - avgTestACC, 2) / n;
                stdTestAUC += Math.pow(er.getAvgTestAUC() - avgTestAUC, 2) / n;
                stdTestBRI += Math.pow(er.getAvgTestBrier() - avgTestBRE, 2) / n;
                stdTrainACC += Math.pow(er.getAvgTrainACC() - avgTrainACC, 2) / n;
                stdTrainAUC += Math.pow(er.getAvgTrainAUC() - avgTrainAUC, 2) / n;
                stdTrainBRI += Math.pow(er.getAvgTrainBrier() - avgTrainBRE, 2) / n;
                stdLength += Math.pow(er.getAvgLength() - avgLength, 2) / n;

                stdBestTestACC += Math.pow(gp.getTestAccError() - avgBestTestACC, 2) / n;
                stdBestTestAUC += Math.pow(gp.getTestAUCError() - avgBestTestAUC, 2) / n;
                stdBestTestBRE += Math.pow(gp.getTestBRIERError() - avgBestTestBRI, 2) / n;
                stdBestTrainACC += Math.pow(gp.getTrainAccError() - avgBestTrainACC, 2) / n;
                stdBestTrainAUC += Math.pow(gp.getTrainAUCError() - avgBestTrainAUC, 2) / n;
                stdBestTrainBRE += Math.pow(gp.getTestBRIERError() - avgBestTrainBRI, 2) / n;
                stdBestLength += Math.pow(gp.getNrOfNodes() - avgBestLength, 2) / n;
            }
            stdTestACC = Math.sqrt(stdTestACC);
            stdTestAUC = Math.sqrt(stdTestAUC);
            stdTrainACC = Math.sqrt(stdTrainACC);
            stdTrainAUC = Math.sqrt(stdTrainAUC);
            stdTestBRI = Math.sqrt(stdTestBRI);
            stdTrainBRI = Math.sqrt(stdTrainBRI);
            stdLength = Math.sqrt(stdLength);

            stdBestTestACC = Math.sqrt(stdBestTestACC);
            stdBestTestAUC = Math.sqrt(stdBestTestAUC);
            stdBestTrainACC = Math.sqrt(stdBestTrainACC);
            stdBestTrainAUC = Math.sqrt(stdBestTrainAUC);
            stdBestTestBRE = Math.sqrt(stdBestTestBRE);
            stdBestTrainBRE = Math.sqrt(stdBestTrainBRE);
            stdBestLength = Math.sqrt(stdBestLength);

            avgs += "\nAVG:" + "\t" + nf.format(avgTrainACC) + "\t" + nf.format(avgTestACC) + "\t" + nf.format(avgTrainAUC) + "\t" + nf.format(avgTestAUC) + "\t" + nf.format(avgTrainBRE) + "\t" + nf.format(avgTestBRE) + "\t" + nf.format(avgLength) + "\n";
            avgs += "STD:" + "\t" + nf.format(stdTrainACC) + "\t" + nf.format(stdTestACC) + "\t" + nf.format(stdTrainAUC) + "\t" + nf.format(stdTestAUC) + "\t" + nf.format(stdTrainBRI) + "\t" + nf.format(stdTestBRI) + "\t" + nf.format(stdLength) + "\n";

            best += "\nAVG:" + "\t" + nf.format(avgBestTrainACC) + "\t" + nf.format(avgBestTestACC) + "\t" + nf.format(getBatchTrainAUC(foldResults)) + "\t" + nf.format(getBatchTestAUC(foldResults)) + "\t" + nf.format(avgBestTrainBRI) + "\t" + nf.format(avgBestTestBRI) + "\t" + nf.format(avgBestLength) + "\n";
            best += "STD:" + "\t" + nf.format(stdBestTrainACC) + "\t" + nf.format(stdBestTestACC) + "\t" + nf.format(stdBestTrainAUC) + "\t" + nf.format(stdBestTestAUC) + "\t" + nf.format(stdBestTrainBRE) + "\t" + nf.format(stdBestTestBRE) + "\t" + nf.format(stdBestLength) + "\n";

            ens += "\nAVG:" + "\t" + nf.format(avgEnsTrainACC) + "\t" + nf.format(avgEnsTestACC) + "\t" + nf.format(getBatchEnsTrainAUC(foldResults)) + "\t" + nf.format(getBatchEnsTestAUC(foldResults)) + "\t" + nf.format(avgEnsTrainBRI) + "\t" + nf.format(avgEnsTestBRI) + "\n";
        } catch (Exception e) {
            e.printStackTrace();
        }
        //Make confusion matrix string
        String cm = "";
        for (int i = 0; i < getData().getNrOfTargetCategories(); i++) {
            cm += getData().getRealTargetClassValue(i) + " (C)\t";
        }
        cm += "\n";
        for (int p = 0; p < getData().getNrOfTargetCategories(); p++) {
            for (int c = 0; c < getData().getNrOfTargetCategories(); c++) {
                cm += confusionMatrix[c][p] + "\t\t";
            }
            cm += getData().getRealTargetClassValue(p) + " (P)";
            cm += "\n";
        }

        return ens + "\n\n" + best + "\n\n" + avgs + "\n\n" + stds + foldSumms + cm;
        // OBS om dessa använda måste PCTrain återställas efteråt eftersom På target och instanser i PCTrain ändras!
        //+ getAllPredictions()+getBatchTrainProbs() + getBatchTestProbs()+getBatchALLProbs();
    }

    public String getAllPredictions() {
        String preds = "";
        String models = "";
        for (EvolutionResult er : foldResults.values()) {
            ArrayList<ArrayList<Prediction>> trainPreds = new ArrayList<ArrayList<Prediction>>();
            ArrayList<ArrayList<Prediction>> testPreds = new ArrayList<ArrayList<Prediction>>();
            String fold = er.getBatchGPs().get(0).getOptions().getTestFOLD();
            models += "\n-------Models for " + fold + " ----------\n";
            for (GP gp : er.getBatchGPs()) {
                models += gp.toString() + "\n";
                trainPreds.add(gp.getOrderedTrain());
                testPreds.add(gp.getOrderedTest());
            }
            trainPreds.add(er.getEnsTrains());
            testPreds.add(er.getEnsTests());
            preds += "-------Train Predictions " + fold + " ---------\n";
            for (int row = 0; row < trainPreds.get(0).size(); row++) {
                for (int col = 0; col < trainPreds.size(); col++) {
                    preds += trainPreds.get(col).get(row).getPrediction() + "\t";
                }
                preds += trainPreds.get(0).get(row).getTargetValue() + "\n";
            }
            preds += "-------Trest Predictions " + fold + " ---------\n";
            for (int row = 0; row < testPreds.get(0).size(); row++) {
                for (int col = 0; col < testPreds.size(); col++) {
                    preds += testPreds.get(col).get(row).getPrediction() + "\t";
                }
                preds += testPreds.get(0).get(row).getTargetValue() + "\n";
            }

        }

        return preds + models;
    }

    public String getAllTestPredictions() {
        NumberFormat nf = NumberFormat.getInstance();
        nf.setMinimumFractionDigits(4);
        String preds = "";
        String models = "";
        for (EvolutionResult er : foldResults.values()) {
            ArrayList<ArrayList<Prediction>> testPreds = new ArrayList<ArrayList<Prediction>>();
            String fold = er.getBatchGPs().get(0).getOptions().getTestFOLD();
            models += "\n-------Models for " + fold + " ----------\n";
            for (GP gp : er.getBatchGPs()) {
                models += gp.toString() + "\n";
                testPreds.add(gp.getOrderedTest());
            }
            testPreds.add(er.getEnsTests());
            preds += "-------Trest Predictions " + fold + " ---------\n";
            for (int row = 0; row < testPreds.get(0).size(); row++) {
                for (int col = 0; col < testPreds.size(); col++) {
                    preds += testPreds.get(col).get(row).getPrediction() + "\t";
                }
                preds += testPreds.get(0).get(row).getTargetValue() + "\n";
            }

        }

        return preds + models;
    }

    public double getPearsonCorrelation(double[] scores1, double[] scores2) {
        double result = 0;
        double sum_sq_x = 0;
        double sum_sq_y = 0;
        double sum_coproduct = 0;
        double mean_x = scores1[0];
        double mean_y = scores2[0];
        for (int i = 2; i < scores1.length + 1; i += 1) {
            double sweep = Double.valueOf(i - 1) / i;
            double delta_x = scores1[i - 1] - mean_x;
            double delta_y = scores2[i - 1] - mean_y;
            sum_sq_x += delta_x * delta_x * sweep;
            sum_sq_y += delta_y * delta_y * sweep;
            sum_coproduct += delta_x * delta_y * sweep;
            mean_x += delta_x / i;
            mean_y += delta_y / i;
        }
        double pop_sd_x = (double) Math.sqrt(sum_sq_x / scores1.length);
        double pop_sd_y = (double) Math.sqrt(sum_sq_y / scores1.length);
        double cov_x_y = sum_coproduct / scores1.length;
        result = cov_x_y / (pop_sd_x * pop_sd_y);
        if (Double.isNaN(result)) {
            return 0;
        }
        return result;
    }

    public String getRegressionResults() {
        NumberFormat nf = NumberFormat.getInstance();
        nf.setMinimumFractionDigits(4);
        String s = "", fs = "";
        fs += "fold\tmse\tlad\tmer\tmre\tz\tmuape\tr2\n";
        String models = "";
        double fmse = 0, flad = 0, fmer = 0, fmre = 0, fz = 0, fmuape = 0, fr2 = 0;
        String testPredictions = "----------TestPredictions-----------\nWeek\tPred\tReal\n";
        String trainPredictions = "----------TrainPredictions-----------\nWeek\tPred\tReal\n";
        int foldNr = 0;
        for (EvolutionResult er : foldResults.values()) {
            foldNr++;
            testPredictions += "Fold: " + foldNr;
            trainPredictions += "Fold: " + foldNr;

            ArrayList<ArrayList<Prediction>> testPreds = new ArrayList<ArrayList<Prediction>>();
            ArrayList<ArrayList<Prediction>> trainPreds = new ArrayList<ArrayList<Prediction>>();
            String fold = er.getBatchGPs().get(0).getOptions().getTestFOLD();
            models += "\n-------Models for " + fold + " ----------\n";
            for (GP gp : er.getBatchGPs()) {
                models += gp.getFormatedTree() + "\n";
                testPreds.add(gp.getOrderedTest());
                trainPreds.add(gp.getOrderedTrain());
            }
            s += fold + "\tmse\tlad\tmer\tmre\tz\tmuape\tr2\n";

            int testRows = testPreds.get(0).size();
            int trainRows = trainPreds.get(0).size();
            double[][] vv = new double[2][testRows];
            // = new double[rows][2];
            for (int c = 0; c < trainPreds.size(); c++) {
                Prediction p;
                trainPredictions += "\n Batch: " + (c + 1) + "\n";
                for (int r = 0; r < trainRows; r++) {
                    p = trainPreds.get(c).get(r);
                    trainPredictions += r + "\t" + p.getPrediction() + "\t" + p.getTargetValue() + "\t" + "\n";
                }
            }


            double bmse = 0, blad = 0, bmer = 0, bmre = 0, bz = 0, bmuape = 0, br2 = 0;
            for (int col = 0; col < testPreds.size(); col++) {
                double mse = 0, lad = 0, mer = 0, mre = 0, z = 0, muape = 0, r2 = 0;
                testPredictions += "\n Batch: " + (col + 1) + "\n";
                Prediction p;
                double estimate, target;
                for (int row = 0; row < testRows; row++) {
                    p = testPreds.get(col).get(row);
                    testPredictions += (trainRows + row) + "\t" + p.getPrediction() + "\t" + p.getTargetValue() + "\t" + "\n";
                    estimate = p.getPrediction() == 0 ? 0.001 : p.getPrediction();
                    target = p.getTargetValue() == 0 ? 0.001 : p.getTargetValue();
                    mse += Math.pow(p.getPrediction() - p.getTargetValue(), 2) / testRows;
                    lad += Math.abs(p.getPrediction() - p.getTargetValue()) / testRows;
                    muape += Math.abs((p.getPrediction() - p.getTargetValue()) / (p.getPrediction() + p.getTargetValue())) / testRows;
                    mer += Math.abs((estimate - target) / (estimate)) / testRows;
                    mre += Math.abs((estimate - target) / (target)) / testRows;
                    z += Math.abs(estimate / target) / testRows;
                    vv[0][row] = p.getPrediction();
                    vv[1][row] = p.getTargetValue();
                }
                r2 = Math.pow(getPearsonCorrelation(vv[0], vv[1]), 2);
                s += "\t" + mse + "\t" + lad + "\t" + mer + "\t" + mre + "\t" + z + "\t" + muape + "\t" + r2 + "\n";
                bmse += mse / testPreds.size();
                blad += lad / testPreds.size();
                bmer += mer / testPreds.size();
                bmre += mre / testPreds.size();
                bmuape += muape / testPreds.size();
                bz += Math.abs(1 - z) / testPreds.size();
                br2 += r2 / testPreds.size();
            }
            fs += fold + "\t" + bmse + "\t" + blad + "\t" + bmer + "\t" + bmre + "\t" + bz + "\t" + bmuape + "\t" + br2 + "\n";
            fmse += bmse / foldResults.size();
            flad += blad / foldResults.size();
            fmer += bmer / foldResults.size();
            fmre += bmre / foldResults.size();
            fmuape += bmuape / foldResults.size();
            fz += bz / foldResults.size();
            fr2 += br2 / foldResults.size();
        }
        fs += "\nMean \t" + fmse + "\t" + flad + "\t" + fmer + "\t" + fmre + "\t" + fz + "\t" + fmuape + "\t" + fr2 + "\n\n";
        return fs + s + models + testPredictions + trainPredictions;
    }

    public String getBatchTrainProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTrain().values().iterator().next().getProbs().length;

        String[] prbs = new String[n * 2 + 2 + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "TrainProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {
                probsAndReals += "F" + f + "\tM" + m + "\tI" + gp.getOrderedTrain().size() + "\n";//.getPcTrain().values().size()+"\n";
                try {

                    for (int i = 0; i < prbs.length; i++) {
                        prbs[i] = "";
                    }
                    for (Prediction p : gp.getOrderedTrain()) {// .getPcTrain().values()) {
                        for (int i = 0; i < n; i++) {
                            prbs[i] += nf.format(p.getProbs()[i]) + "\t";
                        }
                        prbs[n] += p.getPrediction() + "\t";
                        prbs[prbs.length - 1] += p.getTargetValue() + "\t";
                    }
                    for (int i = 0; i < n; i++) {
                        probsAndReals += "P(" + i + ")\t" + prbs[i] + "\n";

                    }
                    probsAndReals += "Pred:\t" + prbs[n] + "\n";

                    probsAndReals += "ACC: " + nf.format(ec.calcError(gp, "TRAIN", "ACC")) + "\n";
                    probsAndReals += "AUC: " + nf.format(ec.calcError(gp, "TRAIN", "AUC")) + "\n";
                    probsAndReals += "VAL: " + nf.format(ec.calcError(gp, "VAL", "ACC")) + "\n";
//------------------------EnsFidelity-----------------------------------------------
                    ArrayList<Prediction> ens = er.getEnsTrains();
                    Prediction eP, gpP;
                    for (int i = 0; i < ens.size(); i++) {
                        eP = ens.get(i);
                        gpP = gp.getPcTrain().get(eP.getInstance());
                        gpP.setTargetValue(eP.getPrediction());
                    }
                    probsAndReals += "EnsFidACC: " + nf.format(ec.calcError(gp, "TRAIN", "ACC")) + "\n";
                    probsAndReals += "EnsFidAUC: " + nf.format(ec.calcError(gp, "TRAIN", "AUC")) + "\n\n";//
//-----------------------------------------------------------------------------------
                } catch (Exception e) {
                    e.printStackTrace();
                }
                m++;
            }
            probsAndReals += "F" + f + " Ens" + "\tI" + er.getBatchGPs().get(0).getOrderedTrain().size() + "\n";//.getPcTrain().values().size()+"\n";
            m = 1;
            ArrayList<Prediction> ensPreds = er.getEnsTrains();
            for (Prediction p : ensPreds) {
                for (int i = 0; i < n; i++) {
                    prbs[n + 2 + i] += nf.format(p.getProbs()[i]) + "\t";
                }
                prbs[prbs.length - 2] += p.getPrediction() + "\t";
            }

            for (int i = n + 2; i < n + 2 + n; i++) {
                probsAndReals += "EnsP(" + (i - n - 2) + ")\t" + prbs[i] + "\n";
            }
            probsAndReals += "EnsPred:\t" + prbs[prbs.length - 2] + "\n";
            probsAndReals += "EnsACC: " + nf.format(er.getEnsTrainACC()) + "\n";
            probsAndReals += "EnsAUC: " + nf.format(er.getEnsTrainAUC()) + "\n";
            probsAndReals += "\nTarget:\t" + prbs[prbs.length - 1] + "\n\n";
            f++;
        }


        return probsAndReals;
    }

    public String getBatchTestProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTest().values().iterator().next().getProbs().length;

        String[] prbs = new String[n * 2 + 2 + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "TestProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {
                probsAndReals += "F" + f + "\tM" + m + "\tI" + gp.getOrderedTest().size() + "\n";//.getPcTrain().values().size()+"\n";
                try {

                    for (int i = 0; i < prbs.length; i++) {
                        prbs[i] = "";
                    }
                    for (Prediction p : gp.getOrderedTest()) {// .getPcTrain().values()) {
                        for (int i = 0; i < n; i++) {
                            prbs[i] += nf.format(p.getProbs()[i]) + "\t";
                        }
                        prbs[n] += p.getPrediction() + "\t";
                        prbs[prbs.length - 1] += p.getTargetValue() + "\t";
                    }
                    for (int i = 0; i < n; i++) {
                        probsAndReals += "P(" + i + ")\t" + prbs[i] + "\n";

                    }
                    probsAndReals += "Pred:\t" + prbs[n] + "\n";

                    probsAndReals += "ACC: " + nf.format(ec.calcError(gp, "TEST", "ACC")) + "\n";
                    probsAndReals += "AUC: " + nf.format(ec.calcError(gp, "TEST", "AUC")) + "\n";
//------------------------EnsFidelity-----------------------------------------------
                    ArrayList<Prediction> ens = er.getEnsTests();
                    Prediction eP, gpP;
                    for (int i = 0; i < ens.size(); i++) {
                        eP = ens.get(i);
                        gpP = gp.getPcTest().get(eP.getInstance());
                        gpP.setTargetValue(eP.getPrediction());
                    }
                    probsAndReals += "EnsFidACC: " + nf.format(ec.calcError(gp, "TEST", "ACC")) + "\n";
                    probsAndReals += "EnsFidAUC: " + nf.format(ec.calcError(gp, "TEST", "AUC")) + "\n\n";//
//-----------------------------------------------------------------------------------
                } catch (Exception e) {
                    e.printStackTrace();
                }
                m++;
            }
            m = 1;

            probsAndReals += "F" + f + " Ens" + "\tI" + er.getBatchGPs().get(0).getOrderedTest().size() + "\n";//.getPcTrain().values().size()+"\n";
            ArrayList<Prediction> ensPreds = er.getEnsTests();
            for (Prediction p : ensPreds) {
                for (int i = 0; i < n; i++) {
                    prbs[n + 2 + i] += nf.format(p.getProbs()[i]) + "\t";
                }
                prbs[prbs.length - 2] += p.getPrediction() + "\t";
            }

            for (int i = n + 2; i < n + 2 + n; i++) {
                probsAndReals += "EnsP(" + (i - n - 2) + ")\t" + prbs[i] + "\n";
            }
            probsAndReals += "EnsPred:\t" + prbs[prbs.length - 2] + "\n";
            probsAndReals += "EnsACC: " + nf.format(er.getEnsTestACC()) + "\n";
            probsAndReals += "EnsAUC: " + nf.format(er.getEnsTestAUC()) + "\n";

            probsAndReals += "\nTarget:\t" + prbs[prbs.length - 1] + "\n\n";
            f++;
        }


        return probsAndReals;
    }

    public String getBatchALLProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTest().values().iterator().next().getProbs().length;

        String[] prbs = new String[n * 2 + 2 + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "\n\nAllProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {

                try {
                    ArrayList<Prediction> all = new ArrayList<Prediction>();
                    PredictionContainer pc = gp.getPcTrain();
                    pc.putAll(gp.getPcTest());

                    all.addAll(er.getEnsTrains());
                    all.addAll(er.getEnsTests());

                    for (int i = 0; i < prbs.length; i++) {
                        prbs[i] = "";
                    }

                    //probsAndReals +="ACC: " + nf.format(sACC.calcACC(gp, gp.getPcTest()))+"\n";
                    //probsAndReals +="AUC: " + nf.format(sAUC.calcAUC(gp, gp.getPcTest()))+"\n";
//------------------------EnsFidelity-----------------------------------------------

                    Prediction eP, gpP;
                    for (int i = 0; i < all.size(); i++) {
                        eP = all.get(i);
                        gpP = pc.get(eP.getInstance());
                        gpP.setTargetValue(eP.getPrediction());
                    }
                    probsAndReals += "F" + f + " M" + m + " I" + all.size() + " AllEnsFidACC: " + nf.format(ec.calcError(pc, "ACC")) + "\n";
                    probsAndReals += "F" + f + " M" + m + " I" + all.size() + " AllEnsFidAUC: " + nf.format(ec.calcError(pc, "AUC")) + "\n";//



//-----------------------------------------------------------------------------------
                } catch (Exception e) {
                    e.printStackTrace();
                }
                m++;
            }
            m = 1;
            f++;
            probsAndReals += "\n";
        }


        return probsAndReals;
    }

    public double getBatchTestAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            ArrayList<Prediction> all = new ArrayList<Prediction>();
            GP gp = null;
            for (EvolutionResult er : foldResults.values()) {
                gp = er.getBestGpFromBatch();
                //  gp.train();
                gp.execute(gp.getPcTest());
                for (Prediction p : gp.getPcTest().values()) {
                    all.add(p);
                }
            }

            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public String getEnsBatchTrainProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTest().values().iterator().next().getProbs().length;

        String[] prbs = new String[n + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "TrainProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {
                probsAndReals += "F" + f + "\tM" + m + "\tI" + gp.getPcTrain().values().size() + "\n";
                try {
                    probsAndReals += "ACC: " + nf.format(gp.calcTrainError("ACC")) + "\n";
                    probsAndReals += "AUC: " + nf.format(gp.calcTrainError("AUC")) + "\n";
                } catch (Exception e) {
                    e.printStackTrace();
                }
                for (int i = 0; i < prbs.length; i++) {
                    prbs[i] = "";
                }
                for (Prediction p : gp.getPcTrain().values()) {
                    for (int i = 0; i < p.getProbs().length; i++) {
                        prbs[i] += nf.format(p.getProbs()[i]) + "\t";
                    }
                    prbs[prbs.length - 2] += p.getPrediction() + "\t";
                    prbs[prbs.length - 1] += p.getTargetValue() + "\t";
                }
                for (int i = 0; i < prbs.length - 2; i++) {
                    probsAndReals += "P(" + i + ")\t" + prbs[i] + "\n";

                }
                probsAndReals += "Prediction:\t" + prbs[prbs.length - 2] + "\n";
                probsAndReals += "Target:\t" + prbs[prbs.length - 1] + "\n";
                m++;
            }
            m = 1;
            f++;
        }
        return probsAndReals;
    }

    public double getBatchEnsTestAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            ArrayList<Prediction> all = new ArrayList<Prediction>();
            Environment env = foldResults.values().iterator().next().getBestGpFromBatch().getEnvironment();
            for (EvolutionResult er : foldResults.values()) {
                all.addAll(er.getEnsTests());
            }
            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public double getBatchEnsTrainAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            List<Prediction> all = new ArrayList<Prediction>();
            Environment env = foldResults.values().iterator().next().getBestGpFromBatch().getEnvironment();
            for (EvolutionResult er : foldResults.values()) {
                all.addAll(er.getEnsTrains());
            }
            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public double getBatchTrainAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            ArrayList<Prediction> all = new ArrayList<Prediction>();
            GP gp = null;
            for (EvolutionResult er : foldResults.values()) {
                gp = er.getBestGpFromBatch();
                for (Prediction p : gp.getPcTrain().values()) {
                    all.add(p);
                }
            }

            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public EvolutionResult getBatchResult(String fold) {
        return foldResults.get(fold);
    }

    public EvolutionResult getLastBatchResult() {
        return foldResults.get(lastTestFold);
    }

    public EvolutionResult getSelectedResult(String fold) {
        return selectedFoldResults.get(fold);
    }

    public void stopEvolution() {
        if (simplyfier != null && !simplyfier.isInterrupted()) {
            simplyfier.interrupt();
        }
        if (t != null) {
            t.interrupt();
        }
    }

    public void startSimplification() {
        try {
            simplyfier = new Simplyfier(bestGP, world, simplificationListeners);
            simplyfier.start();
            simplyfier.join();
            // simplyfier.simplify();
            bestSimplyfiedGP = simplyfier.getSimplyfiedGP();
        } catch (Exception e) {
            ErrorManager.getInstance().reportError(e);
        }
    }

    public void createAlternativeSolution(GP gp) {
        try {
            world.getEnvironment().getOptions().setSIMPLIFICATION_FUNCTION("AlternativeSolutionFitness");
            world.getEnvironment().getOptions().setGp(gp); //Sätter i GP i den Genrealla Options som används när nya GP skapas
            gp.getOptions().setGp(gp); //Sätter GP i det aktuella GPt option (olika eftersom varje GP måste komm ihåg sina inställningar (fold).
            simplyfier = new Simplyfier(gp, world, simplificationListeners);
            simplyfier.start();
//            simplyfier.join();
            //simplyfier.start();
            bestSimplyfiedGP = simplyfier.getSimplyfiedGP();
            world.getEnvironment().getOptions().setRUN_CREATE_ALTERNATIVE(false);
        } catch (Exception e) {
            ErrorManager.getInstance().reportError(e);
        }
    }

    public void startSimplification(GP gp) {
        try {
            world.getEnvironment().getOptions().setRUN_SIMPLIFICATION(true);
         //   world.getEnvironment().getOptions().setSIMPLIFICATION_FUNCTION("SimplificationFunction");
            simplyfier = new Simplyfier(gp, world, simplificationListeners);
            simplyfier.start();
//            simplyfier.join();
            //simplyfier.start();
            bestSimplyfiedGP = simplyfier.getSimplyfiedGP();
            world.getEnvironment().getOptions().setRUN_SIMPLIFICATION(false);
        } catch (Exception e) {
            ErrorManager.getInstance().reportError(e);
        }
    }

    public GP getBestGpFromBatch() {
        return getBestGpFromBatch(batchGPs);
    }

    public GP getBestGpFromBatch(ArrayList<GP> gps) {
        GP best;
        best = null;
        for (GP current : gps) {
            if (best == null || best.getFitness() > current.getFitness()) {
                best = current;
            }
        }
        if (best != null) {
            best = (GP) best.clone();
            best.clean();
        }
        return best;
    }

    public double getAverageLength() {
        return world.getAvgPopulationLength();
    }

    public double getAverageFitness() {
        return world.getAvgPopulationFitness();
    }

    public GP getBestGP() {
        return bestGP;
    }

    public int getCurrentGeneration() {
        return currentGeneration;
    }

    public int getCurrentBatch() {
        return currentBatch + 1;
    }

    public int getPersistence() {
        return persistence;
    }

    private synchronized void fireGenerationTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.generationTerminated(this);
            }
        }
    }

    private synchronized void fireEvolutionTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.evolutionTerminated(this);
            }
        }
    }

    private synchronized void fireBatchTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.batchTerminated(this);
            }
        }
    }

    private synchronized void fireEvolutionStarted() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.evolutionStarted(this);
            }
        }
    }

    private synchronized void fireEvolutionProcessTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.evolutionProcessTerminated(this);
            }
        }
        t = null;
        //  this.world=null;
        // this.data.destroyDistanceMatrixAndLookup();
    }

    public String getSummary() {
        return summary;
    }

    public String getSummaryForSelectedGPs() {
        return selectedSummary;
    }

    /**
     * @return the options
     */
    public Options getOptions() {
        return options;
    }

    /**
     * @return the data
     */
    public ArffTableModel getData() {
        return data;
    }

    /**
     * @return the environment
     */
    public Environment getEnvironment() {
        return environment;
    }

    public World getWorld() {
        return world;
    }
}
